from __future__ import division
import math

print "First, I have to say that it is a very big step move."
print "Let's do them"
print "sd_ordered_set_BM25"
print "fd_ordered_set_BM25"
print "sd_unordered_set_BM25"
print "BM25_related"
print "sd_ordered_set_LM"
print "fd_ordered_set_LM"
print "sd_unordered_set_LM"
print "fd_unordered_set_LM"


def LM_related(mode,size,trecID,queryVariations,queryVariationContentAndVariantIDDict_in_function,queryVariantIDTrecIDAndItsParametersTfDicts,queryVariantIDAndItsParametersCfDicts,trecIDAndWithDocLength):
    finalAccessKey = mode + str(size)
    # print "finalAccessKey:",finalAccessKey
    
    score = 0.0
    MIOU = 2000
    collection_total_num_docs = 67167 # This number comes from the subset of all the documents which have been evaluated.
    
    for currentQueryVariant in queryVariations:
        
        accessKeyForCf = queryVariationContentAndVariantIDDict_in_function[currentQueryVariant]            
        accessKeyForTf = str(accessKeyForCf) + "_" + trecID
        
        if accessKeyForTf in queryVariantIDTrecIDAndItsParametersTfDicts:
            tf = queryVariantIDTrecIDAndItsParametersTfDicts[accessKeyForTf][finalAccessKey]
        else:
            tf = 0.0
        
        cf = queryVariantIDAndItsParametersCfDicts[accessKeyForCf][finalAccessKey]

        
        # print "currentQueryVariant:",currentQueryVariant
        # print "accessKeyForCf:",accessKeyForCf
        # print "accessKeyForTf:",accessKeyForTf
        # print "docLength",trecIDAndWithDocLength[trecID]
        # print "cf:",cf
        # print "tf:",tf
            
        # This part is doing the LM feature
        partial_LM_component_1 = tf + MIOU * (cf / collection_total_num_docs)
        partial_LM_component_2 = trecIDAndWithDocLength[trecID] + MIOU
        
        # print "partial_LM_component_1:",partial_LM_component_1
        # print "partial_LM_component_2:",partial_LM_component_2
        
        if partial_LM_component_1 != 0:
            # Here, the base is 10
            partial_LM_score = math.log(partial_LM_component_1 / partial_LM_component_2)
        else:
            # Known problem. Updated by Wei: 2012/10/22
            # In this case, the semantics have some problem. Mostly because there are small window size like 1, but have large number of query variants like the size of 3 or 4 
            partial_LM_score = -9999
        # print "partial_LM_score:",partial_LM_score
        # print 
        score += partial_LM_score

    return score


def BM25_related(mode,size,trecID,queryVariations,queryVariationContentAndVariantIDDict_in_function,queryVariantIDTrecIDAndItsParametersTfDicts,queryVariantIDAndItsParametersDfDicts,trecIDAndWithDocLength):

    finalAccessKey = mode + str(size)
    #print "finalAccessKey:",finalAccessKey
    
    
    score = 0.0
    
    # BM25 parameters: see 'http://en.wikipedia.org/wiki/Okapi_BM25'.
    kBm25K1 =  2.0  # k1
    kBm25B = 0.75   #b
    
    # We can precompute a few of the BM25 values here.
    kBm25NumeratorMul = kBm25K1 + 1
    kBm25DenominatorAdd = kBm25K1 * (1 - kBm25B)
    collection_average_doc_len = 1922.5 # This number comes from the subset of all the documents which have been evaluated.
    collection_total_num_docs = 67167 # This number comes from the subset of all the documents which have been evaluated.
    
    kBm25DenominatorDocLenMul = kBm25K1 * kBm25B / collection_average_doc_len 
    
    
    
    for currentQueryVariant in queryVariations:
        
        accessKeyForDf = queryVariationContentAndVariantIDDict_in_function[currentQueryVariant]

                        
        accessKeyForTf = str(accessKeyForDf) + "_" + trecID
        
        

        
        idf_t = math.log10(1 + (collection_total_num_docs - queryVariantIDAndItsParametersDfDicts[ accessKeyForDf ][finalAccessKey] + 0.5) / (queryVariantIDAndItsParametersDfDicts[ accessKeyForDf ][finalAccessKey] + 0.5))
        if accessKeyForTf in queryVariantIDTrecIDAndItsParametersTfDicts:
            tf = queryVariantIDTrecIDAndItsParametersTfDicts[accessKeyForTf][finalAccessKey]
        else:
            tf = 0.0
        

        
        
        partial_BM25 = idf_t * ( tf * kBm25NumeratorMul) / (  tf + kBm25DenominatorAdd + kBm25DenominatorDocLenMul * trecIDAndWithDocLength[trecID] )
        

        score += partial_BM25
        '''
        print "currentQueryVariant:",currentQueryVariant
        print "accessKeyForDf:",accessKeyForDf
        print "accessKeyForTf:",accessKeyForTf
        print "similar idf:",queryVariantIDAndItsParametersDfDicts[ accessKeyForDf ][finalAccessKey]
        print "docLength",trecIDAndWithDocLength[trecID]
        print "real idf_t:",idf_t
        print "tf:",tf
        print "partial_BM25:",partial_BM25        
        '''
    return score
    





# deal with final-150Clueweb09Queries-variations.txt
print "Deal with final-150Clueweb09Queries-variations.txt"
queryIDAndQueryVariationsDict = {}

tempInputFileName = "/data5/team/weijiang/the_new_trip_of_feature_generation/final-150Clueweb09Queries-variations.txt"
tempInputFileHandler = open(tempInputFileName,"r")
for line in tempInputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    queryID = int( lineElements[0].split("_")[0] )
    #print "queryID:",queryID
    if queryID not in queryIDAndQueryVariationsDict:
        queryIDAndQueryVariationsDict[queryID] = lineElements[1:]
        
#print queryIDAndQueryVariationsDict

tempInputFileHandler.close()


# deal with final-150Clueweb09Queries-variations-only.txt
print "Deal with final-150Clueweb09Queries-variations-only.txt"
print "dict:queryVariationContentAndVariantIDDict key:int:queryVariantID value:string:queryVariantContent"
queryVariationContentAndVariantIDDict = {}
tempInputFileName = "/data5/team/weijiang/the_new_trip_of_feature_generation/final-150Clueweb09Queries-variations-only.txt"
tempInputFileHandler = open(tempInputFileName,"r")
for line in tempInputFileHandler.readlines():
    lineElements = line.strip().split(":")
    
    queryVariantID = int( lineElements[0] )
    queryVariantContent = lineElements[1]
    
    if queryVariantContent not in queryVariationContentAndVariantIDDict:
        queryVariationContentAndVariantIDDict[queryVariantContent] = queryVariantID

tempInputFileHandler.close()

# deal with final-complexFeaturesSupportedFileBothModeForTf_dicts.txt
print "Deal with final-complexFeaturesSupportedFileBothModeForTf_dicts.txt"
print "dict:queryVariantIDTrecIDAndItsParametersTfDicts key:string:queryVariantID + '_' + trecID value:dict:the inner dict has keys[M_1, M_4, M_8, M_32, N_ordered_1, N_ordered_4, N_ordered_8, N_ordered_32, N_unordered_1, N_unordered_4, N_unordered_8, N_unordered_32]"
queryVariantIDTrecIDAndItsParametersTfDicts = {}
tempInputFileName = "/data5/team/weijiang/the_new_trip_of_feature_generation/final-complexFeaturesSupportedFileBothModeForTf_dicts.txt"
tempInputFileHandler = open(tempInputFileName,"r")
tempInputFileHandler.readline()
for line in tempInputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    
    queryVariantID = lineElements[0]
    trecID = lineElements[1]
    
    key = queryVariantID + "_" + trecID
    
    currentDict = {}
    currentDict["M_1"] = int(lineElements[2])
    currentDict["M_4"] = int(lineElements[3])
    currentDict["M_8"] = int(lineElements[4])
    currentDict["M_32"] = int(lineElements[5])
    
    currentDict["N_ordered_1"] = int(lineElements[6])
    currentDict["N_ordered_4"] = int(lineElements[7])
    currentDict["N_ordered_8"] = int(lineElements[8])
    currentDict["N_ordered_32"] = int(lineElements[9])
    
    currentDict["N_unordered_1"] = int(lineElements[10])
    currentDict["N_unordered_4"] = int(lineElements[11])
    currentDict["N_unordered_8"] = int(lineElements[12])
    currentDict["N_unordered_32"] = int(lineElements[13])    
    
    if key not in queryVariantIDTrecIDAndItsParametersTfDicts:
        queryVariantIDTrecIDAndItsParametersTfDicts[key] = currentDict
    else:
        print "error"
        exit(1)
tempInputFileHandler.close()
print "queryVariantIDTrecIDAndItsParametersTfDicts['6_clueweb09-en0000-00-00142']:",queryVariantIDTrecIDAndItsParametersTfDicts['6_clueweb09-en0000-00-00142']
print "queryVariantIDTrecIDAndItsParametersTfDicts['550_clueweb09-enwp03-59-01036']:",queryVariantIDTrecIDAndItsParametersTfDicts['550_clueweb09-enwp03-59-01036']
     
        
        
        
# deal with final-complexFeaturesSupportedFileBothModeForDf_dicts.txt
print "Deal with final-complexFeaturesSupportedFileBothModeForDf_dicts.txt"
print "dict:queryVariantIDAndItsParametersDfDicts key:int:queryVariantID value:dict:the inner dict has keys[M_1, M_4, M_8, M_32, N_ordered_1, N_ordered_4, N_ordered_8, N_ordered_32, N_unordered_1, N_unordered_4, N_unordered_8, N_unordered_32]"
queryVariantIDAndItsParametersDfDicts = {}
tempInputFileName = "/data5/team/weijiang/the_new_trip_of_feature_generation/final-complexFeaturesSupportedFileProductionForDf_dicts.txt"
tempInputFileHandler = open(tempInputFileName,"r")
tempInputFileHandler.readline()
for line in tempInputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    
    queryVariantID = int(lineElements[0])
    
    key = queryVariantID
    
    currentDict = {}
    currentDict["M_1"] = int(lineElements[2])
    currentDict["M_4"] = int(lineElements[3])
    currentDict["M_8"] = int(lineElements[4])
    currentDict["M_32"] = int(lineElements[5])
    
    currentDict["N_ordered_1"] = int(lineElements[6])
    currentDict["N_ordered_4"] = int(lineElements[7])
    currentDict["N_ordered_8"] = int(lineElements[8])
    currentDict["N_ordered_32"] = int(lineElements[9])
    
    currentDict["N_unordered_1"] = int(lineElements[10])
    currentDict["N_unordered_4"] = int(lineElements[11])
    currentDict["N_unordered_8"] = int(lineElements[12])
    currentDict["N_unordered_32"] = int(lineElements[13])    
    
    if key not in queryVariantIDAndItsParametersDfDicts:
        queryVariantIDAndItsParametersDfDicts[key] = currentDict
    else:
        print "error"
        exit(1)
tempInputFileHandler.close()
        
        
# deal with final-complexFeaturesSupportedFileBothModeForCf_dicts.txt
print "Deal with final-complexFeaturesSupportedFileBothModeForCf_dicts.txt"
print "dict:queryVariantIDAndItsParametersCfDicts key:int:queryVariantID value:dict:the inner dict has keys[M_1, M_4, M_8, M_32, N_ordered_1, N_ordered_4, N_ordered_8, N_ordered_32, N_unordered_1, N_unordered_4, N_unordered_8, N_unordered_32]"
queryVariantIDAndItsParametersCfDicts = {}
tempInputFileName = "/data5/team/weijiang/the_new_trip_of_feature_generation/final-complexFeaturesSupportedFileProductionForCf_dicts.txt"
tempInputFileHandler = open(tempInputFileName,"r")
tempInputFileHandler.readline()
for line in tempInputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    
    queryVariantID = int(lineElements[0])
    
    key = queryVariantID
    
    currentDict = {}
    currentDict["M_1"] = int(lineElements[2])
    currentDict["M_4"] = int(lineElements[3])
    currentDict["M_8"] = int(lineElements[4])
    currentDict["M_32"] = int(lineElements[5])
    
    currentDict["N_ordered_1"] = int(lineElements[6])
    currentDict["N_ordered_4"] = int(lineElements[7])
    currentDict["N_ordered_8"] = int(lineElements[8])
    currentDict["N_ordered_32"] = int(lineElements[9])
    
    currentDict["N_unordered_1"] = int(lineElements[10])
    currentDict["N_unordered_4"] = int(lineElements[11])
    currentDict["N_unordered_8"] = int(lineElements[12])
    currentDict["N_unordered_32"] = int(lineElements[13])    
    
    if key not in queryVariantIDAndItsParametersCfDicts:
        queryVariantIDAndItsParametersCfDicts[key] = currentDict
    else:
        print "error"
        exit(1)
tempInputFileHandler.close()



# deal with final-all_human_judged_webpage_with_doc_length_in_words.txt
print "Deal with final-all_human_judged_webpage_with_doc_length_in_words.txt"
print "dict:trecIDAndWithDocLength key:string:trecID value:int:docLength"
trecIDAndWithDocLength = {}
tempInputFileName = "/data5/team/weijiang/the_new_trip_of_feature_generation/final-all_human_judged_webpage_with_doc_length_in_words.txt"
tempInputFileHandler = open(tempInputFileName,"r")
for line in tempInputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    
    trecID = lineElements[0]
    docLength = int(lineElements[1])
    
    key = trecID   
    
    if key not in trecIDAndWithDocLength:
        trecIDAndWithDocLength[key] = docLength
    else:
        print "error"
        exit(1)
tempInputFileHandler.close()
#print "trecIDAndWithDocLength['clueweb09-en0000-00-00142']",trecIDAndWithDocLength["clueweb09-en0000-00-00142"]


# main loop
modesList = ["M_","N_ordered_","N_unordered_"]
sizesList = [1,4,8,32]

inputFileName = "/data5/team/weijiang/the_new_trip_of_feature_generation/wholeTrainingFileTempleteV3_phase0.txt.input"
inputFileHandler = open(inputFileName,"r")

outputFileName = "/data5/team/weijiang/the_new_trip_of_feature_generation/wholeTrainingFileTempleteV3_output_feature_new_phase2.txt"
outputFileHandler = open(outputFileName,"w")
outputFileHandler.write("queryID" + " " + "trecID" + " " + "BM25_M_1" + " " + "BM25_M_4" + " " + "BM25_M_8" + " " + "BM25_M_32" + " " + "BM25_N_ordered_1" + " " + "BM25_N_ordered_4" + " "+ "BM25_N_ordered_8" + " " + "BM25_N_ordered_32" + " " + "BM25_N_unordered_1" + " " + "BM25_N_unordered_4" + " " + "BM25_N_unordered_8" + " " + "BM25_N_unordered_32" + " ")
outputFileHandler.write("LM_M_1" + " " + "LM_M_4" + " " + "LM_M_8" + " " + "LM_M_32" + " " + "LM_N_ordered_1" + " " + "LM_N_ordered_4" + " "+ "LM_N_ordered_8" + " " + "LM_N_ordered_32" + " " + "LM_N_unordered_1" + " " + "LM_N_unordered_4" + " " + "LM_N_unordered_8" + " " + "LM_N_unordered_32")
outputFileHandler.write("\n")

for index,line in enumerate(inputFileHandler.readlines()):
    # in production mode, comment out the following two lines
    # if index == 1:
    #     break
    lineElements = line.strip().split(" ")
    queryID = int(lineElements[0])
    trecID = lineElements[1]

    outputLine = str(queryID) + " " + trecID + " "
    for mode in modesList:
        for size in sizesList:
            if queryID != 70:
                returnedScore = BM25_related(mode,size,trecID,queryIDAndQueryVariationsDict[queryID],queryVariationContentAndVariantIDDict,queryVariantIDTrecIDAndItsParametersTfDicts,queryVariantIDAndItsParametersDfDicts,trecIDAndWithDocLength)
            else:
                returnedScore = 0.0
            outputLine += str(returnedScore) + " "
    
    for mode in modesList:
        for size in sizesList:
            if queryID != 70:
                returnedScore = LM_related(mode,size,trecID,queryIDAndQueryVariationsDict[queryID],queryVariationContentAndVariantIDDict,queryVariantIDTrecIDAndItsParametersTfDicts,queryVariantIDAndItsParametersCfDicts,trecIDAndWithDocLength)
            else:
                returnedScore = 0.0
            outputLine += str(returnedScore) + " "
    
    print outputLine
    outputFileHandler.write(outputLine + "\n")

inputFileHandler.close()
outputFileHandler.close()

